#!/usr/bin/env python
# encoding: utf-8
"""
TwitterCollectorRealTime.py

Created by Marcel Caraciolo on 2009-11-26.
Copyright (c) 2009 Federal University of Pernambuco. All rights reserved.
"""

__author__ = 'caraciol@gmail.com'
__version__ = '0.1'


from optparse import OptionParser
import random
import sqlite3
import time

try:
  import twitterT
except:
	print "error:\tcan't import python-twitter module, you must install the module:\n"
	exit()


screenname = ""
usersList = []
TOTAL_REQUESTS = 0
FRIENDS_MAX = 20
STATUSES_MAX  =  600
DB_NAME = "twitterFinalDB.db"



def getStatuses(api,user,statusLimit = STATUSES_MAX ):
	""" Get the statuses from the user 
	Parameters:
		api: The api connector for python-twitter
		user: the user object
		statusLimit: the number of statuses to fetch
	Return:
		statusList: the list of statuses
	"""
	global TOTAL_REQUESTS
	screenName = user.GetScreenName()
	page = 1
	statusFlag = True
	statusList = []
	print screenName
	while statusFlag:
		for i in range(5):
			try:
				statuses = api.GetUserTimeline(screenName,page=page,count=200)
				TOTAL_REQUESTS+=1
				break
			except:
				print " FAILED user " + str(screenName) + ", retrying"
				time.sleep(4)
		if len(statuses) > 0 and len(statusList) < statusLimit:
			tempList = [(status.GetId(),screenName,status.GetCreatedAt(),status.GetRelativeCreatedAt(), status.GetInReplyToScreenName(), status.GetFavorited(), status.GetText()) for status in statuses]
			statusList.extend(tempList)
			page+=1
		else:
			statusFlag = False
	return statusList



def openDb(databaseFile):
	""" Method to  open a database
		Parameters:
			databaseFile: The database name
		Return:
			the pointer to database
	"""
	conn = sqlite3.connect(databaseFile)
	return conn


def saveStatusesToDatabase(userStatuses,usersStatsList,databaseFile=DB_NAME):
	""" Method to save the statuses information at the database
		Parameters:
			userStatuses: the user statuses
			usersStatsList: the list of statuses of the users
			databaseFile: the database file path
		Return:
			None
	"""
	print "Saving the statuses data ... "
	conn = openDb(databaseFile)
	cursor = conn.cursor()
	
	#Drop all data from the table
	st = "drop table if exists twitterStatuses"
	cursor.execute(st)
	conn.commit()
	
	#Create the table
	stmt = "Create table if not exists twitterStatus (identify integer, screenName text, created_at text, relative_created_at text, reply_to_screen_name text, favorited bool, message text)"
	cursor.execute(stmt)
	conn.commit()
	
	#Load the statuses data
	usersStatsList.append(userStatuses)
	
	for statuses in usersStatsList:
		stmt = "insert into twitterStatus values (?, ?, ?, ?, ?, ?, ?) "
		cursor.executemany(stmt,statuses)
	conn.commit()
	cursor.close()
	conn.close()
	print "Loading the data set %s ... finished! " % databaseFile
	


def saveUsersToDatabase(user,usersList,databaseFile=DB_NAME):
	""" Method to save the users information at the database
		Parameters:
			user: the user info
			usersList: the friends info
			databaseFile: The database file path
		Return:
			None
	"""
	print "Saving all user data... "
	conn = openDb(databaseFile)
	cursor = conn.cursor()
	
	#Drop all data from the table
	st = "drop table if exists twitterUsers"
	cursor.execute(st)
	conn.commit()
	
	#Create the table
	stmt = "Create table if not exists twitterUsers (identify integer, screenName text, name text, location text, timezone text,  description text, followers_count integer, friends_count integer , statuses_count integer, favourites_count integer, friends text)"
	cursor.execute(stmt)
	conn.commit()
	
	#Load the user data 
	usersList.append(user)
	for user in usersList:
		stmt = "insert into twitterUsers values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
		cursor.execute(stmt, (user.GetId(), user.GetScreenName(), user.GetName(), user.GetLocation(),user.GetTimeZone(), user.GetDescription(), user.GetFollowersCount(), user.GetFriendsCount(), user.GetStatusesCount(), user.GetFavouritesCount(), ""))
	conn.commit()
	cursor.close()
	conn.close()
	print "Loading the data set %s ... finished! " % databaseFile

		


def getFriendsInfo(api,usersList,numberOfFriends=FRIENDS_MAX):
	global TOTAL_REQUESTS
	""" Method to get about information of the user's friends
		Parameters:
			api : The twitter api handler
			usersList:  The user ids list
			numberOfFriends: The number of Friends to get
		Return:
			friends : Returns the friends info.
	"""
	friends = []
	#Shuffle the usersList
	random.shuffle(usersList)
	#Get numberofFriends from the list
	if usersList:
		if type(usersList[0]) == type(1): #Is an id ?
			friends = [api.GetUser(nid) for nid in usersList[0:numberOfFriends]]
			TOTAL_REQUESTS+=numberOfFriends
		else: #Is an object ?
			friends = [user for user in usersList[0:numberOfFriends]]
			TOTAL_REQUESTS+=1
	return friends
	

if __name__ == "__main__":
	print "Twitter User Recommendations Tool "
	print "By Marcel Pinheiro Caraciolo (@marcelcaraciolo)"
	print "http://aimotion.blogspot.com\n"
	
	parser = OptionParser()
	parser.add_option("-u", "--username", dest="username", 
						help="The twitter username", metavar="USERNAME")
	parser.add_option("-p", "--password", dest="password",
						help="The twitter password (optional)", metavar="PASSWORD")
	
	(options, args) = parser.parse_args()
	
	
	if (not options.username):
		parser.print_help()
		exit()
	
	
	pass_msg = "authenticated" if options.password else "unauthenticated"
	
	print "Starting the Twitter User Recommendation for user %s (%s) ... " % (options.username, pass_msg)
	
	screename = options.username
	api = twitterT.Api(username=options.username, password=options.password)
	TOTAL_REQUESTS+=1
	
	print "\nFirst Step: Collect user data..."
	print "Collecting basic information about the user %s ... " % (options.username,)
	#Getting info about the user 
	user = api.GetUser(screename)
	TOTAL_REQUESTS+=1
	#Getting info about friendIds (Bug 01: Work on with more 5000 thousand friends!)
    #                             (Bug 02: Impossible to fetch more than 150 friends with userIDs)
  	#usersList = api.GetFriends(screenname)
	usersList = api.GetFriendsIds(screenname)
	TOTAL_REQUESTS+=1
	usersList = getFriendsInfo(api,usersList[:])
	print "Loaded friends. Total: %s " % str(len(usersList))
	
	#Save all the data in the database
	saveUsersToDatabase(user,usersList[:])
	
	#Get my statuses
	print "Collecting my statuses..."
	myStatuses =  getStatuses(api,user)
	print "Statuses collected. Total: %s" % str(len(myStatuses))
	
	#Get the statuses of the users	
	print "Collecting friends statuses..."
	userStatuses = [getStatuses(api,friend) for friend in usersList]

	print "Statuses collected. Total: %s" % str(sum([len(statuses) for statuses in userStatuses]))

	#Saving the statuses on the database
	saveStatusesToDatabase(myStatuses,userStatuses[:])
	
	
	print "\nSecond Step: Generating the symbolic user profile data..."
	print "Creating symbolic profile for user %s "  % (screename,)
	userProfile = {}
	
	
	
	
	friendsProfile = {}
	
	
	
	
	
	
	#1 - pego meu user Ok!
	#2 - pego meu profile basico Ok!
	#3 - pego meus n ultimos twitters Ok!
	#4 - pego meus x amigos  ok!
	#5 - pego os  n posts dos meus amigos ok!
	#5 - Profiles simbolicos
	#6 - clusterizo e vejo amigos que tem maior ranking
	#4:13 7 - pego esse x amigos do ranking e olho os amigos deles
	#8 - amigos em comum x  k twitters desses amigos -> compara ocm o meu 
	#9 -  amigos em comum de mior quantidade com k twitters desses amigos comparados com o meu
	#gera um ranking
	#10 - apresenta os l novos users
	#com maior score
	#pra mim
	#esse eh o resultado.
	
	

	
	
	
	
	